"""
file:spiderwin_ctl.py

author:nxm

describe:

time:2017-8-2
"""
# -*- coding: utf-8 -*-
from PyQt4 import QtCore, QtGui

import gui.xueqcrawl_ui,sqlite3
from bs4 import BeautifulSoup
import lxml, urllib.request, urllib.error, json, re, requests, time
from requests import exceptions
import threading
import http.cookiejar

class XueqCrawlWindow_UI(QtGui.QMainWindow, gui.xueqcrawl_ui.Ui_XueqCrawlWindow):
    xueqtodalysignal = QtCore.pyqtSignal()
    xueqtextBrowersignal = QtCore.pyqtSignal(str)
    headers = {
        'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'h-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4',
        'Connection': 'keep-alive',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Host': 'xueqiu.com',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36',
        'X-Requested-With': 'XMLHttpRequest'
    }
    def __init__(self):
        super(QtGui.QMainWindow, self).__init__()  # 调用父类的init函数
        super(gui.xueqcrawl_ui.Ui_XueqCrawlWindow, self).__init__()  # 调用父类的init函数
        self.setupUi(self)

         # 设置窗口图标
        icon = QtGui.QIcon()
        icon.addPixmap(QtGui.QPixmap(":/icon/toolkit.ico"))
        self.setWindowIcon(icon)

        self.xueqButton_1.clicked.connect(self.XueqButtonProcess)
        self.xueqClearButton.clicked.connect(self.XueqClearButtonProcess)
        self.xueqButton_3.clicked.connect(self.XueqButton3Process)

        self.xueqloginButton.clicked.connect(self.XueLoginButtonProcess)
        self.xueqoutloginButton.clicked.connect(self.XueqOutLoginButtonProcess)
        self.xueqhsButton.clicked.connect(self.XueqHSButtonProcess)

        self.xueqtodaylist = []
        self.cookie = []

        self.xueqtodalysignal.connect(self.XueqTodayListDisplay)
        self.xueqtextBrowersignal.connect(self.XueqTextBrowerDisplay)

        self.session = requests.session()
        self.xuequid = 0

    def XueqButtonProcess(self):
        self.xueqtextBrowser.append("<span>开始数据采集</span>")
        spiderthread = threading.Thread(target=self.XueqSpiderRun)
        spiderthread.start()

    def XueqClearButtonProcess(self):
        self.xueqtextBrowser.clear()


    def XueqButton3Process(self):
        pass


    def XueqHSButtonProcess(self):
        self.xueqtextBrowser.append('<span> 开始采集股票信息</span>')
        page = self.xueqpagespinBox.value()
        hsthread = threading.Thread(target=self.XueqHStockRun, args=(page,))
        hsthread.start()


    def XueLoginButtonProcess(self):
        log = self.XueqLogin(self.xuequserEdit.text(), self.xueqpasswordEdit.text())

        if "error_description" in log.text:
            self.xueqtextBrowser.append("<span> 登录失败</span>")
        else:
            logjson = json.loads(log.text)
            self.xuequid = logjson['uid']

            self.xueqtextBrowser.append("<span>User:%s Uid:%d 登录成功</span>"%(self.xuequserEdit.text(),self.xuequid))
            self.xuequserEdit.setEnabled(False)
            self.xueqpasswordEdit.setEnabled(False)

    def XueqOutLoginButtonProcess(self):
        url = 'https://xueqiu.com/user/logout?redirect_uri=/'
        logout = self.session.get(url, headers=self.headers)

        if logout.status_code == 200:
            self.xueqtextBrowser.append("<span> 退出成功</span>")
            self.xuequid = 0
            self.xuequserEdit.setEnabled(True)
            self.xueqpasswordEdit.setEnabled(True)
        else:
            self.xueqtextBrowser.append("<span> 退出失败</span>")


    def XueqLogin(self,user,password):
        url = 'https://xueqiu.com/snowman/login'
        # 构造 Request headers
        data = {'remember_me':'true','username':user,'password':password}
        s = self.session.post(url, data=data,headers=self.headers,timeout = 5)

        return s



    def XueqTodayListDisplay(self):
        self.xueqtextBrowser.append('<span>采集文章数量%s</span>'%(str(len(self.xueqtodaylist))))
        '''
        conn = sqlite3.connect("test.db")
        cu = conn.cursor()
        cu.execute("drop table if exists todaylist ")
        conn.commit()
        cu.execute("create table if not exists todaylist(TITLE text,TITLE_URL text, CONTENT text)")
        '''
        for listval in self.xueqtodaylist:
            destr = listval["des"]
            if '<img' in destr:
                destr = re.sub('<img.*?>',' ',destr)

            #cu.execute("insert into todaylist( TITLE, TITLE_URL,CONTENT) values ('%s','%s','%s')"%(listval['title'], "https://xueqiu.com"+listval['target'], destr))
            self.xueqtextBrowser.append("<span>标题：</span>"+"<a href=\"%s\"><span>%s</span></a>"%("https://xueqiu.com"+listval['target'],listval['title']))
            self.xueqtextBrowser.append("<span>简介：</span>"+destr)
            self.xueqtextBrowser.append("<span>**********************************************************************</spin>")
        #conn.commit()
       # cu.close()
        #conn.close()

    def XueqTextBrowerDisplay(self, distr):
        self.xueqtextBrowser.append("<span>%s</span>"%(distr))


    def XueqSpiderRun(self):
        readlen = self.xueqcntspinBox.value()
        self.xueqtodaylist = []
        nextmaxid = -1

        while readlen != 0 :
            if readlen >= 20:
                html = self.getRequestHtml(nextmaxid, 20)
                readlen -= 20
            else:
                html = self.getRequestHtml(nextmaxid, readlen)
                readlen = 0

            if html != []:
                readlist = json.loads(html)
                nextmaxid = readlist['next_max_id']
                for content in readlist['list']:
                    data = json.loads(content['data'])
                    outdic = {'title': data['topic_title'], 'des': data['description'], 'target': data['target']}
                    self.xueqtodaylist.append(outdic)

                if readlen == 0:
                    self.xueqtodalysignal.emit()
            else:
                self.xueqtextBrowser.append("<span>采集错误</span>")

    def XueqHStockRun(self,pagelen):

        conn = sqlite3.connect("test.db")
        cu = conn.cursor()
        try:
            cu.execute("drop table if exists %s "%(time.strftime('GP_%Y_%m_%d')))
            conn.commit()                                    #股票代码    股票名字     股票市值             涨跌幅            市盈率      成交额
            cu.execute("create table if not exists %s(symbol text,name text, marketcapital double, percent double, pettm text, amount  text,orgtype text, majorbiz text, companyurl text)"%(time.strftime('GP_%Y_%m_%d')))
        except:
            cu.close()
            conn.close()
            print("数据库操作异常")

        for page in range(pagelen):
            html = self.getHStockHtml(page+1)
            if html != []:
                hsjson = json.loads(html)['stocks']
                for readval in hsjson:
                    try:
                        if readval['pettm'] != ''and readval['pettm'] != '0':#对数据判断，把异常数据清除
                            companyhtml = self.getCompanyInfo(readval['symbol'])
                            if companyhtml != []:
                                companyinfo = json.loads(companyhtml)['tqCompInfo']

                            val = float(readval['marketcapital'])/100000000
                            cu.execute("insert into %s( symbol, name, marketcapital, percent, pettm, amount, orgtype, majorbiz, companyurl) values ('%s','%s', %s, %s,'%s','%s','%s', '%s','%s')" %(time.strftime('GP_%Y_%m_%d'),readval['symbol'],  readval['name'],  str(val),  readval['percent'],  readval['pettm'], readval['amount'], companyinfo['orgtype'],companyinfo['majorbiz'],companyinfo['compurl']))

                    except:
                        print("数据库操作异常", readval)
                conn.commit()
            self.xueqtextBrowersignal.emit("第%s页采集完成" % (str(page+1)))
            time.sleep(0.1)

        cu.close()
        conn.close()
        self.xueqtextBrowersignal.emit('<span> 股票信息采集结束</span>')

    def saveGdInfo(self, symbol, company, pstr):

        readhtml = self.getZYGDHJson(symbol)

        if readhtml != []:
            readjson = json.loads(readhtml)
            opentxt = open(".\gd\%s_%s.txt"%(symbol,company),'w')
            opentxt.write(" 股票代号：%s   股东名单:%s    " % (symbol, company))
            opentxt.write(pstr + "\n\n\n")
            opentxt.write("股东名称：       持股比例：        \n")

            for listval in readjson["list"][:3]:

                gdlist = listval["list"]
                opentxt.write("---------------时间：%s---------------\n" % (listval["enddate"]))

                for gdval in gdlist:
                    try:
                        opentxt.write("%s            %s%%           \n"%(gdval['shholdername'], gdval['holderrto']))
                    except:
                        print(gdval)

            opentxt.close()



    def getRequestHtml(self,maxid,count):

        if self.cookie == []:
            self.cookie = self.getCookie()

        try:
            url = 'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id=%s&count=%s&category=-1'%(str(maxid), str(count))
            html = requests.get(url, headers=self.headers, cookies=self.cookie).content.decode('utf-8')
            return html
        except:
            print("request html get error")
           # self.cookie = self.getCookie()
            return []

    def getCompanyInfo(self, symbol):
        try:
            url = 'https://xueqiu.com/stock/f10/compinfo.json?symbol=%s&page=1&size=4&_=1504310169600' % (symbol)
            html = requests.get(url, headers=self.headers, cookies=self.cookie).content.decode('utf-8')
            return html
        except:
            print("request html get error")
            # self.cookie = self.getCookie()
            return []


    def getHStockHtml(self, page):

        if self.cookie == []:
            self.cookie = self.getCookie()

        try:
            url = 'https://xueqiu.com/stock/cata/stocklist.json?page=%d&size=90&order=desc&orderby=percent&type=11%%2C12&_=12321552'%(page)
            html = requests.get(url, headers=self.headers, cookies=self.cookie).content.decode('utf-8')
            return html
        except:
            print("request html get error")
            #self.cookie = self.getCookie()
            return []

    def getZYGDHJson(self, symbol):

        if self.cookie == []:
            self.cookie = self.getCookie()

        try:
            url = 'https://xueqiu.com/stock/f10/shareholder.json?symbol=%s&page=1&size=4&_=254125633'%(symbol)
            html = requests.get(url, headers=self.headers, cookies=self.cookie).content.decode('utf-8')
            return html
        except:
            print("request html get error")
            #self.cookie = self.getCookie()
            return []

    def getCookie(self):
        cookie = http.cookiejar.CookieJar()
        handler = urllib.request.HTTPCookieProcessor(cookie)
        opener = urllib.request.build_opener(handler)
        opener.addheaders = [('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36')]
        try:
            opener.open('https://xueqiu.com/')
            return cookie
        except:
            print("get cookie error")
            return []


    def getHtmlPage(self, count):
        try:
            url = 'https://xueqiu.com/v4/statuses/public_timeline_by_category.json?since_id=-1&max_id=-1&count=%s&category=-1'%(str(count))
            headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36',
                       'Cookie':" aliyungf_tc=AQAAAHdGaVyjEgEACnQJd8/5ra8G3hft; s=fq11gog8gv; webp=0; __utma=1.1960059648.1490936436.1492519024.1495241507.3; __utmc=1; __utmz=1.1490936436.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); device_id=545adc2eff10ad442c4956b4c49cff56; xq_a_token=35967e90fece12b70f15096c72ae9b6982f628a7; xq_a_token.sig=-qUrmkra84xJqFAD2ZukbCZ1FMA; xq_r_token=04a34d441044eea56430a435d6c270f709b923ae; xq_r_token.sig=jIYThmOvEthGQpyKL58Zz9dXhE8; u=901503192124090; Hm_lvt_1db88642e346389874251b5a1eded6e3=1503105060; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1503192126"
            }
            request =  urllib.request.Request(url,headers=headers)
            response = urllib.request.urlopen(request)
            html = response.read().decode('utf-8')

            return html
        except urllib.error.HTTPError as e:
            #QtGui.QMessageBox.warning(None, '错误', "HTTP异常", QtGui.QMessageBox.Ok)
            print(e.code)
            return []
        except urllib.error.URLError as e:
           # QtGui.QMessageBox.warning(None, '错误', "URL异常", QtGui.QMessageBox.Ok)
            print(e.reason)
            return []
        except:
            print("采集异常")
            return []


    def closeEvent(self, event):
        pass